#include "battleship.h"
#include "mcts.h"
#include "network.h"
#include "pocman.h"
#include "rocksample.h"
#include "swinput.h"
#include "swtiger.h"
#include "tag.h"
#include "experiment.h"
#include <boost/program_options.hpp>
#include "boost/timer.hpp"

/////////////////////////////////////////////
// added from IPPC 2011
int runIPPC(MCTS::PARAMS sp, string instanceName, string host, string clientName, int port, double timePerAction);
int connectToServer(const char *hostname, int port);
void doMainClientLoop(const std::string& instance_name, const std::string& client_name, int fd, MCTS::PARAMS sp, double timePerAction);
// added from IPPC 2011
/////////////////////////////////////////////
/////////////////////////////////////////////

using namespace std;
using namespace boost::program_options;

void UnitTests()
{
	cerr << "Testing UTILS" << endl;
	UTILS::UnitTest();
	cerr << "Testing COORD" << endl;
	COORD::UnitTest();
	cerr << "Testing MCTS" << endl;
	MCTS::UnitTest();
}

void disableBufferedIO(void)
{
	setbuf(stdout, NULL);
	setbuf(stdin, NULL);
	setbuf(stderr, NULL);
	setvbuf(stdout, NULL, _IONBF, 0);
	setvbuf(stdin, NULL, _IONBF, 0);
	setvbuf(stderr, NULL, _IONBF, 0);
}

int main(int argc, char* argv[])
{
	MCTS::PARAMS searchParams;
	EXPERIMENT::PARAMS expParams;
	SIMULATOR::KNOWLEDGE knowledge;

	string problem, outputfile, policy;
	string instanceName, host, clientName;
	int port;
	double timePerAction;

	int size, number, treeknowledge = 1, rolloutknowledge = 1, smarttreecount = 10;
	double smarttreevalue = 1.0;

	options_description desc("Allowed options");
	desc.add_options()
        								("help", "produce help message")
        								("test", "run unit tests")
        								("problem", value<string>(&problem), "problem to run")
        								("outputfile", value<string>(&outputfile)->default_value("output.txt"), "summary output file")
        								("policy", value<string>(&policy), "policy file (explicit POMDPs only)")
        								("size", value<int>(&size), "size of problem (problem specific)")
        								("number", value<int>(&number), "number of elements in problem (problem specific)")
        								("timeout", value<double>(&expParams.TimeOut), "timeout (seconds)")
        								("mindoubles", value<int>(&expParams.MinDoubles), "minimum power of two simulations")
        								("maxdoubles", value<int>(&expParams.MaxDoubles), "maximum power of two simulations")
        								("runs", value<int>(&expParams.NumRuns), "number of runs")
        								("accuracy", value<double>(&expParams.Accuracy), "accuracy level used to determine horizon")
        								("horizon", value<int>(&expParams.UndiscountedHorizon), "horizon to use when not discounting")
        								("num steps", value<int>(&expParams.NumSteps), "number of steps to run when using average reward")
        								("verbose", value<int>(&searchParams.Verbose), "verbosity level")
        								("autoexploration", value<bool>(&expParams.AutoExploration), "Automatically assign UCB exploration constant")
        								("exploration", value<double>(&searchParams.ExplorationConstant), "Manual value for UCB exploration constant")
        								("usetransforms", value<bool>(&searchParams.UseTransforms), "Use transforms")
        								("transformdoubles", value<int>(&expParams.TransformDoubles), "Relative power of two for transforms compared to simulations")
        								("transformattempts", value<int>(&expParams.TransformAttempts), "Number of attempts for each transform")
        								("userave", value<bool>(&searchParams.UseRave), "RAVE")
        								("ravediscount", value<double>(&searchParams.RaveDiscount), "RAVE discount factor")
        								("raveconstant", value<double>(&searchParams.RaveConstant), "RAVE bias constant")
        								("treeknowledge", value<int>(&knowledge.TreeLevel), "Knowledge level in tree (0=Pure, 1=Legal, 2=Smart)")
        								("rolloutknowledge", value<int>(&knowledge.RolloutLevel), "Knowledge level in rollouts (0=Pure, 1=Legal, 2=Smart)")
        								("smarttreecount", value<int>(&knowledge.SmartTreeCount), "Prior count for preferred actions during smart tree search")
        								("smarttreevalue", value<double>(&knowledge.SmartTreeValue), "Prior value for preferred actions during smart tree search")
        								("disabletree", value<bool>(&searchParams.DisableTree), "Use 1-ply rollout action selection")
        								("instancename", value<string>(&instanceName), "IPPC Competition - instance name")
        								("host", value<string>(&host), "IPPC Competition - host")
        								("clientname", value<string>(&clientName), "IPPC Competition - clientname")
        								("port", value<int>(&port), "IPPC Competition - port")
        								("timeperaction", value<double>(&timePerAction), "IPPC Competition - timeperaction")
        								;

	variables_map vm;
	store(parse_command_line(argc, argv, desc), vm);
	notify(vm);

	if (vm.count("help"))
	{
		cerr << desc << "\n";
		return 1;
	}

	if (vm.count("problem") == 0)
	{
		cerr << "No problem specified" << endl;
		return 1;
	}

	if (vm.count("test"))
	{
		cerr << "Running unit tests" << endl;
		UnitTests();
		return 0;
	}

	SIMULATOR* real = 0;
	SIMULATOR* simulator = 0;

	if (problem == "battleship")
	{
		real = new BATTLESHIP(size, size, number);
		simulator = new BATTLESHIP(size, size, number);
	}
	else if (problem == "pocman")
	{
		real = new FULL_POCMAN;
		simulator = new FULL_POCMAN;
	}
	else if (problem == "network")
	{
		real = new NETWORK(size, number);
		simulator = new NETWORK(size, number);
	}
	else if (problem == "rocksample")
	{
		real = new ROCKSAMPLE(size, number);
		simulator = new ROCKSAMPLE(size, number);
	}
	else if (problem == "swinput")
	{
		real = new SWINPUT();
		simulator = new SWINPUT();
	}
	else if (problem == "swtiger")
	{
		real = new SWTIGER();
		simulator = new SWTIGER();
	}
	else if (problem == "tag")
	{
		real = new TAG(number);
		simulator = new TAG(number);
	}
	else if (problem == "IPPC")
	{
		runIPPC(searchParams, instanceName, host, clientName, port, timePerAction);
		cerr << "Finish Running IPPC" << endl;
		exit(0);
	}else  if (problem == "testing") {

		cerr << "swinput";
		simulator = new SWINPUT();
		MCTS mcts(*simulator, searchParams);
		int action = mcts.SelectAction(10.0);
		cerr << action;

//		cerr << "swtiger";
//		//simulator = new ROCKSAMPLE(7, 7);
//		simulator = new SWTIGER();
//		MCTS mcts2(*simulator, searchParams);
//		int action2 = mcts2.SelectAction(10.0);
//		cerr << action2;

		exit(0);
	}
	else
	{
		cerr << "Unknown problem" << endl;
		exit(1);
	}

	simulator->SetKnowledge(knowledge);
	EXPERIMENT experiment(*real, *simulator, outputfile, expParams, searchParams);
	experiment.DiscountedReturn();

	delete real;
	delete simulator;
	return 0;
}

//////////////////////////////////////////////
/////////////////////////////////////////////
//added for IPPC

int runIPPC(MCTS::PARAMS sp, string instance_request, string host, string clientName, int port, double timePerAction) {


	// SYLMOD: changed the hardcoded arguments to commandline arguments.
	// Later, this should be changed to input arguments, read from the scheduler

	/* we hardcode most arguments, better to read from command line **/
	//std::string host("localhost");
	//std::string client_name("your-client-name");
	//int port = 2323;

	//std::cerr << "echo command-line arguments: "<< host << " " << client_name << " " << port << " " << instance_request << std::endl;

	try {

		int socket = connectToServer(host.c_str(), port);
		if (socket <= 0) {
			std::cerr << "Could not connect to " << host << ':' << port << std::endl;
			return 1;
		}

		doMainClientLoop(instance_request, clientName, socket, sp, timePerAction);

	} catch (const std::exception& e) {
		std::cerr << std::endl << "mdpclient: " << e.what() << std::endl;
		return 1;
	} catch (...) {
		std::cerr << "mdpclient: fatal error" << std::endl;
		return -1;
	}

	return 0;




}

/*
 * Copyright 2003-2005 Carnegie Mellon University and Rutgers University
 * Copyright 2007 Hakan Younes
 * Copyright 2011 Sungwook Yoon, Scott Sanner (modified for RDDLSim)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/***********************************************************************
 * Note: the procedures in this file go from least important (top) to  *
 * most important (bottom), so read in reverse to get a sense of       *
 * control flow.                                                       *
 ***********************************************************************/

/***********************************************************************/
/*                           INCLUDES / DEFS                           */
/***********************************************************************/

#include "strxml.h"

/** Ubuntu wants the following two libraries **/
#include <stdlib.h>
#include <string.h>
/**********************************************/

#include <list>  // SYLMOD

struct ObsAndVal {  	// SYLMOD  <--- NOT SURE IF THIS IS THE BEST struct TO USE
	std::string ObsName;
	std::string ObsValString;
} ;


#include <string>
#include <iostream>
#include <cerrno>
#include <cstdio>
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <netinet/in.h>

#if HAVE_SSTREAM
#include <sstream>
#else
#include <strstream>
namespace std {
typedef std::ostrstream ostringstream;
}
#endif
#include <unistd.h>

/***********************************************************************/
/*                RDDLSIM SERVER INTERACTION HELPER PROCS              */
/***********************************************************************/

/* Extracts a fluent from the given XML node. */
static int getFluent(const XMLNode* atomNode, std::string& fluent, std::string& valueString) {

	if (atomNode == 0 || atomNode->getName() != "observed-fluent") {
		return -1;
	}

	// Get fluent name
	if (!atomNode->dissect("fluent-name", fluent)) {
		return -1;
	}

	// check if fluent name includes a hyphen and replace with underscore
	// find all the positions in fluent that has hyphen
	size_t found;
	int countLoop = 0; // keeps track of whether is first time through the loop
	while(1) {
		if (countLoop == 0) {
			found=fluent.find("-");
			if (found!=std::string::npos) {
				//std::cerr << int(found) << std::endl;
				fluent.replace(found,1,"_");  // it's ok to do this because "-" and "_" are the same length
			} else break;
		} else {
			found=fluent.find("-", found+1);
			if (found!=std::string::npos) {
				//std::cerr << int(found) << std::endl;
				fluent.replace(found,1,"_");
			} else break;
		}
		countLoop++;
	}


	// Get fluent arguments and value
	fluent += "__"; //fluent += "(";  // SYLMOD
	int arg_count = 0;
	bool valueDummy;
	for (int i = 0; i < atomNode->size(); i++) {
		const XMLNode* termNode = atomNode->getChild(i);
		if (termNode == 0) {
			continue;

		} else if (termNode->getName() == "fluent-arg") {
			if (arg_count++ > 0)
				fluent += "_";   // fluent += ",";  // SYLMOD
			fluent += termNode->getText();

		} else if ( termNode != 0 && termNode->getName() == "fluent-value") {
			//value = termNode->getText() == "true";  // SYLMODIFIED_FOR_TESTING
			valueString = termNode->getText();
		}
	}
	//fluent += ")";  // SYLMODIFIED_FOR_TESTING

	return (int)valueDummy; // Should be -1:error, 0:false, 1:true   // SYLMODIFIED_FOR_TESTING
}

/* Extracts a state (multiple fluents/values) from the given XML node. */
static bool showState(const XMLNode* stateNode,  vector<int>&listVals, list<ObsAndVal>& ListObsAndVals) {
	if (stateNode == 0) {
		return false;
	}
	if (stateNode->getName() != "turn") {
		return false;
	}

	std::cerr << "==============================================\n" << std::endl;

	ObsAndVal tempRec;

	if (stateNode->size() == 2 &&
			stateNode->getChild(1)->getName() == "no-observed-fluents") {

		// The first turn for a POMDP will have this null observation
		std::cerr << "No state/observations received.\n" << std::endl;
		listVals.push_back(0);
	} else {


		//vector<int> listVals;
		// Show all state or observation fluents for this turn
		//std::cerr << "True state/observation variables:" << std::endl;
		for (int i = 0; i < stateNode->size(); i++) {
			const XMLNode* cn = stateNode->getChild(i);
			if (cn->getName() == "observed-fluent") {
				//std::string fluent;
				//int val = getFluent(cn, fluent);


				std::string fluent;
				std::string valueString;
				int val = getFluent(cn, fluent, valueString);


				if (valueString.compare("true") == 0)
					listVals.push_back(1);
				else
					listVals.push_back(2);


				tempRec.ObsName = fluent;
				tempRec.ObsValString = valueString;
				ListObsAndVals.push_back (tempRec);

				//// Only display true fluents
				//if (val)
				//  std::cerr << "- " << fluent << std::endl;
			}
		}
		//std::cerr << std::endl;
	}

	return true;
}

/* Sends an action on the given stream. */
void sendAction(std::ostream& os, std::string actionString) {

	// first check if the action is noop
	if (actionString.compare("noop") == 0) {

		// A simple 'noop'
		std::cerr << "--> Action taken: noop\n" << std::endl;
		os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
				<< "<actions/>" << '\0';

		return;
	}

	// SYLMOD, code to break up action string into component parts - action, action-name, action-arg
	// check if actionString includes "___" triple underscores
	// find all the positions in actionString that has "___"

	std::list<std::string> ListActions;
	std::string subString; //temp substring

	size_t found;
	size_t lastfound;
	int countLoop = 0; // keeps track of whether is first time through the loop
	while(1) {
		if (countLoop == 0) {
			found=actionString.find("___");
			if (found!=std::string::npos) {

				subString.clear();
				for (int i=0; i < int(found); i++)
				{
					subString+= actionString[i];

				}

				ListActions.push_back (subString);
				lastfound = found;

			} else break;

		} else {

			found=actionString.find("___", found+1);
			if (found!=std::string::npos) {

				subString.clear();
				for (int i = int(lastfound)+3; i < int(found); i++)
				{
					subString+= actionString[i];

				}

				ListActions.push_back (subString);

				lastfound = found;
			} else break;
		}
		countLoop++;
	}

	if (countLoop > 0) // this means we found at least one "___"
	{
		subString.clear();
		for (int i = int(lastfound)+3; i < actionString.length(); i++)
		{
			subString+= actionString[i];

		}

		ListActions.push_back (subString);
	} else  // the entire actionString is a single action
	{
		subString.clear();
		for (int i = 0; i < actionString.length(); i++)
		{
			subString+= actionString[i];

		}

		ListActions.push_back (subString);

	}

	std::cerr << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
			<< "<actions>"  << std::endl;
	os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
	os << "<actions>";

	std::string actionArgsString;

	for (std::list<std::string>::iterator it = ListActions.begin(); it != ListActions.end(); it++) {

		std::cerr     << "<action>"  << std::endl;
		os << "<action>" ;

		found=(*it).find("__");  // there must be one and only one
		if (found!=std::string::npos) {  // break up the action into two parts: action-name and action-args

			subString.clear();
			for (int i=0; i < int(found); i++)
			{
				subString+= (*it)[i];
			}
			std::cerr<< "<action-name>" << subString <<  "</action-name>" << std::endl;
			os << "<action-name>" << subString <<  "</action-name>";
			lastfound = found;

		} else  std::cerr << "Error in format for action : " << *it << std::endl;  //print error message

		// deal with action-args
		actionArgsString.clear();
		for (int i = int(lastfound)+2; i < (*it).length(); i++)
		{
			actionArgsString+= (*it)[i];
		}


		countLoop = 0;
		while(1) {
			if (countLoop == 0) {
				found=actionArgsString.find("_");
				if (found!=std::string::npos) {

					subString.clear();
					for (int i=0; i < int(found); i++)
					{
						subString+= actionArgsString[i];

					}
					std::cerr<< "<action-arg>" << subString <<  "</action-arg>" << std::endl;
					os << "<action-arg>" << subString <<  "</action-arg>";

					lastfound = found;

				} else break;

			} else {

				found=actionArgsString.find("_", found+1);
				if (found!=std::string::npos) {

					subString.clear();
					for (int i = int(lastfound)+1; i < int(found); i++)
					{
						subString+= actionArgsString[i];

					}
					std::cerr<< "<action-arg>" << subString <<  "</action-arg>" << std::endl;
					os << "<action-arg>" << subString <<  "</action-arg>";

					lastfound = found;
				} else break;
			}
			countLoop++;
		}

		if (countLoop > 0) // this means we found at least one "_"
		{
			subString.clear();
			for (int i = int(lastfound)+1; i < actionArgsString.length(); i++)
			{
				subString+= actionArgsString[i];

			}
			std::cerr<< "<action-arg>" << subString <<  "</action-arg>" << std::endl;
			os << "<action-arg>" << subString <<  "</action-arg>";


		} else  // the entire actionString is a single action
		{
			subString.clear();
			for (int i = 0; i < actionArgsString.length(); i++)
			{
				subString+= actionArgsString[i];

			}
			std::cerr<< "<action-arg>" << subString <<  "</action-arg>" << std::endl;
			os << "<action-arg>" << subString <<  "</action-arg>";

		}



		std::cerr<< "<action-value>true</action-value>" << std::endl;
		std::cerr     << "</action>"  << std::endl;
		os<< "<action-value>true</action-value>" ;
		os     << "</action>"  ;


	}


	std::cerr   << "</actions>" << '\0' << std::endl;
	os << "</actions>" << '\0';

	// A simple 'noop'
	//std::cerr << "--> Action taken: noop\n" << std::endl;
	//os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
	//   << "<actions/>" << '\0';

	//SYLMOD commented out temporarily to focus on reading-in and formatting of observations
	// A more complex action example specific to 'SysAdmin'
	//int comp_num = (rand() % 4) + 1;
	//  int comp_num = (rand() % 10) + 1;
	//  std::cerr << "--> Action taken: reboot(c" << comp_num << ")\n" << std::endl;
	//  os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
	//     << "<actions><action>"
	//     << "<action-name>reboot</action-name>"
	//     << "<action-arg>c" << comp_num << "</action-arg>"
	//     << "<action-value>true</action-value>"
	//     << "</action></actions>" << '\0';

	////  std::cerr << "--> Action taken: noop\n" << std::endl;
	////  os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
	////     << "<actions></actions>" << '\0';

	// Example for a domain with three *concurrent* single-argument actions
	//os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
	//   << "<actions>"
	//   << "<action><action-name>advance</action-name><action-arg>i1</action-arg><action-value>true</action-value></action>"
	//   << "<action><action-name>advance</action-name><action-arg>i2</action-arg><action-value>true</action-value></action>"
	//   << "<action><action-name>advance</action-name><action-arg>i3</action-arg><action-value>true</action-value></action>"
	//   << "</actions>" << '\0';
}

/* Extracts session request information. */
static bool sessionRequestInfo(const XMLNode* node,
		int& rounds, long& time, int& sessionId) {

	if (node == 0) {
		return false;
	}

	std::string s;
	if (!node->dissect("num-rounds", s)) {
		return false;
	}
	rounds = atoi(s.c_str());

	if (!node->dissect("time-allowed", s)) {
		return false;
	}
	time = atol(s.c_str());

	//SYLMOD added to test out information in session initialization
	if (!node->dissect("session-id", s)) {
		return false;
	}
	sessionId = atoi(s.c_str());

	return true;
}

/* Extracts round request information. */
static bool roundRequestInfo(const XMLNode* node,
		int& round_number, long& time_left, int& rounds_left) {

	if (node == 0) {
		return false;
	}

	std::string s;

	if (!node->dissect("time-left", s)) {
		return false;
	}
	time_left = atol(s.c_str());


	if (!node->dissect("round-num", s)) {
		return false;
	}
	round_number = atoi(s.c_str());


	if (!node->dissect("round-left", s)) {
		return false;
	}
	rounds_left = atoi(s.c_str());


	return true;
}

/* helper connect function */
int connectToServer(const char *hostname, int port)
{
	struct hostent *host = ::gethostbyname(hostname);
	if (!host) {
		perror("gethostbyname");
		return -1;
	}

	int sock = ::socket(PF_INET, SOCK_STREAM, 0);
	if (sock == -1) {
		perror("socket");
		return -1;
	}

	struct sockaddr_in addr;
	addr.sin_family=AF_INET;
	addr.sin_port=htons(port);
	addr.sin_addr = *((struct in_addr *)host->h_addr);
	memset(&(addr.sin_zero), '\0', 8);

	if (::connect(sock, (struct sockaddr*)&addr, sizeof(addr)) == -1) {
		perror("connect");
		return -1;
	}
	return sock;
	//remember to call close(sock) when you're done
}

/***********************************************************************/
/*                     MAIN SERVER INTERACTION LOOP                    */
/***********************************************************************/

/* Constructs an XML client and actually runs all of the server interaction*/
void doMainClientLoop(const std::string& instance_name,
		const std::string& client_name,
		int fd,
		MCTS::PARAMS sp,
		double timePerAction) {
	std::ostringstream os;
	os.str("");
	os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" << "<session-request>"
			<<  "<problem-name>" << instance_name << "</problem-name>"
			<<  "<client-name>" << client_name << "</client-name>"
			<<  "<no-header/>"
			<< "</session-request>"
			<< '\0';
#if !HAVE_SSTREAM
	os << '\0';
#endif
	write(fd, os.str().c_str(), os.str().length());

	std::list<ObsAndVal> ListObsAndVals; //SYLMOD added to hold the observations read-in from the server
	vector<int>listVals;

	const XMLNode* sessionInitNode = read_node(fd);

	int total_rounds;
	long round_time;
	int sessionId;  //SYLMOD added

	if (!sessionRequestInfo(sessionInitNode,
			total_rounds, round_time, sessionId)) {
		std::cerr << "Error in server's session-request response" << std::endl;
		if (sessionInitNode != 0) {
			delete sessionInitNode;
		}
		return;
	}

	if (sessionInitNode != 0) {
		delete sessionInitNode;
	}

	std::cerr << "echo session init information" << std::endl;
	std::cerr << "sessionId : " << sessionId << " total rounds : " << total_rounds << " round time : " << round_time << std::endl;


	//total_rounds = 1;

	// Do a round
	int rounds_count = 0;

	while (++rounds_count <= total_rounds) {

		//SYLMOD added to keep track of number of actions
		//int actions_count = 0;

		std::cerr << "***********************************************" << std::endl;
		std::cerr << ">>> ROUND INIT " << rounds_count << "/" << total_rounds << "; time remaining = " << round_time << std::endl;
		std::cerr << "***********************************************" << std::endl;

		os.str("");
		os << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" <<  "<round-request/>" << '\0';
#if !HAVE_SSTREAM
		os << '\0';
#endif
		write(fd, os.str().c_str(), os.str().length());
		const XMLNode* roundInitNode = read_node(fd);

		int round_number, rounds_left;
		long time_left;


		if (!roundInitNode || roundInitNode->getName() != "round-init" || !roundRequestInfo(roundInitNode,
				round_number, time_left, rounds_left)) {
			std::cerr << "Error in server's round-request response" << std::endl;
			if (roundInitNode != 0) {
				delete roundInitNode;
			}
			return;
		}

		delete roundInitNode;

		std::cerr << "echo round init information" << std::endl;
		std::cerr << "round number : " << round_number  << " time left : " << time_left << " rounds left : " <<     rounds_left << std::endl;


		const XMLNode* response = 0;




		SWINPUT* simulator = 0;
		simulator = new SWINPUT();
		SIMULATOR::KNOWLEDGE knowledge;
		knowledge.TreeLevel = 1;
		knowledge.RolloutLevel = 1;
		knowledge.SmartTreeCount = 10;
		knowledge.SmartTreeValue = 1.0;
		simulator->SetKnowledge(knowledge);

		sp.ExplorationConstant = simulator->GetRewardRange();
		MCTS::InitFastUCB(sp.ExplorationConstant);
		MCTS mcts(*simulator, sp);

		while (1) {

			boost::timer timer;

			if (response != 0) {
				delete response;
			}

			response = read_node(fd);

			if (!response) {
				std::cerr << "Invalid state response" << std::endl;
				return;
			}

			if (response->getName() == "round-end"
					|| response->getName() == "session-end") {

				std::string s;
				if (response->dissect("round-reward", s)) {
					float reward = atof(s.c_str());
					std::cerr << "***********************************************" << std::endl;
					std::cerr << ">>> END OF ROUND -- REWARD RECEIVED: " << reward << std::endl;
					//std::cerr << "***********************************************\n" << std::endl;
				}

				if (response->dissect("time-used", s)) {
					long time_used  = atol(s.c_str());
					std::cerr << ">>> time used: " << time_used << std::endl;
				}

				if (response->dissect("turns-used", s)) {
					int turns_used  = atoi(s.c_str());
					std::cerr << ">>> turns used: " << turns_used << std::endl;
				}

				std::cerr << "***********************************************\n" << std::endl;

				//std::cerr << response << std::endl;
				break;
			}

			// Display the state / observations
			// TODO: Read this into your planner's internal representation

			ListObsAndVals.clear(); //SYLMOD clear the list
			listVals.clear();

			if (!showState(response,listVals,ListObsAndVals)) {
				std::cerr << "Invalid state response: " << response << std::endl;
				delete response;
				return;
			}

			int action;
			int obsValue;

			if (listVals.front() != 0) {

				std::cerr << "Observations received: ";
				for(int i = 0; i < listVals.size(); i++) {
					std::cerr << " " << listVals[i];
				}
				std::cerr << " " << std::endl;
				std::cerr << " " << std::endl;

				obsValue = simulator->convertToFlatObsState(listVals);
				std::cerr << "obsValue: " << obsValue << endl;

				mcts.Update(action,obsValue, 0);
			}


			for (std::list<ObsAndVal>::iterator it = ListObsAndVals.begin(); it != ListObsAndVals.end(); it++)
				std::cerr << (*it).ObsName << " " << (*it).ObsValString << " " << std::endl;

			//std::cerr << std::endl;

			// Send an action
			// TODO: Based on the state above, your planner chooses the action

			action = mcts.SelectAction(timePerAction);

			std:: string actionString = simulator->actionNames.at(action);

			std::cerr << action << endl;
			std::cerr << actionString << endl;

			//std::string testAction = "noop";

			//SYLMOD added to keep track of number of actions
			//actions_count++;
			//std::cerr << "Action number: " << actions_count << std::endl;
			os.str("");
			sendAction(os, actionString);

#if !HAVE_SSTREAM
			os << '\0';
#endif
			write(fd, os.str().c_str(), os.str().length());

		}


		if (response && response->getName() == "end-session") {
			delete response;
			break;
		}

		if (response != 0) {
			delete response;
		}
	}
	const XMLNode* endSessionNode = read_node(fd);


	if (endSessionNode) {

		std::string s;
		if (endSessionNode->dissect("total-reward", s)) {
			float reward = atof(s.c_str());
			std::cerr << "***********************************************" << std::endl;
			std::cerr << ">>> END OF SESSION -- OVERALL REWARD: " << reward << std::endl;
			//std::cerr << "***********************************************\n" << std::endl;
		}


		if (endSessionNode->dissect("time-used", s)) {
			long time_Used = atol(s.c_str());
			std::cerr << ">>> time used: " << time_Used  << std::endl;
		}

		if (endSessionNode->dissect("rounds-used", s)) {
			int rounds_used = atoi(s.c_str());
			std::cerr << ">>> rounds used: " << rounds_used  << std::endl;
		}

		std::cerr << "***********************************************\n" << std::endl;

		//std::cerr << endSessionNode << std::endl;
		delete endSessionNode;
	}
}
